# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
import datetime

from itemadapter import ItemAdapter
import pymysql


class DoubanSpiderPipeline:
    def open_spider(self, spider):
        self.conn = pymysql.connect(
            host='localhost',
            port=3306,
            user='root',
            password='111111',
            database='movie',
            charset='utf8'
        )
        self.cursor = self.conn.cursor()
        self.data = []

    def process_item(self, item, spider):
        if spider.name == 'douban':
            is_new = '1'
        elif spider.name == 'douban_newmovie':
            is_new = '2'
        else:
            is_new = '0'
        title = item.get('title', '')
        types = item.get('types', '')
        regions = item.get('regions', '')
        actors = item.get('actors', '')
        cover_url = item.get('cover_url', '')
        score = item.get('score', '')
        release_date = item.get('release_date', '')
        run_time = item.get('run_time', '')
        review_number = item.get('review_number', '')
        content = item.get('content', '').strip().replace('\n', '').replace('\r', '').replace("'", "")

        self.data.append(
            (title, types, regions, actors, cover_url, score, release_date, run_time, review_number, content, is_new))
        print('每一个data：',self.data)
        if len(self.data) == 5:
            self._write_db()
            self.data.clear()
        return item

    def close_spider(self, spider):
        if len(self.data) > 0:
            self._write_db()
        print('数据库已经写入完成')
        print('爬虫已结束：', spider.name)
        self.cursor.close()
        self.conn.close()

    def _write_db(self):
        sql = 'INSERT INTO douban_movie(`title`,types,regions,actors,`cover_url`,`score`,`release_date`,`run_time`,`review_number`,`description`,`is_new`)' \
              'values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
        self.cursor.executemany(sql, self.data)
        print('正在写入数据库')
        self.conn.commit()
